package com.yjjxt

import org.apache.spark.rdd.RDD
import org.apache.spark.{SparkConf, SparkContext}

object Hello11Distinct {
  def main(args: Array[String]): Unit = {
    val sparkContext = new SparkContext((new SparkConf().setMaster("local").setAppName("Join" + System.currentTimeMillis())))
    val array1 = Array[String]("Hello1 user1", "Hello1 user2", "Hello user", "Hello user")
    val lines1 = sparkContext.parallelize(array1, 4)
    val words1 = lines1.map(ele => (ele.split(" ")(0), ele.split(" ")(1)))

    //对去重后
    val lines11: RDD[String] = lines1.distinct()
    val words11: RDD[(String, String)] = words1.distinct()
    //打印分区的数目
    println("分区数目Lines=>" + lines11.getNumPartitions)
    println("分区数目Words=>" + words11.getNumPartitions)

    //开始打印
    lines11.foreach(ele => println("lines11=>" + ele))
    words11.foreach(ele => println("words11=>" + ele))

    println("---------------------------")

    //查看Tuple的Distinct
    sparkContext.parallelize(Array[String]("hello", "hello")).map((_, Math.random())).distinct().foreach(println)

  }
}
